# -*- coding:utf-8 -*-
# @Time: 2021/12/10 21:24
# @Author : Saki;
# @File : test1.py
# @Software : PyCharm
from django.test import TestCase
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import pandas as pd
import requests
from collections import defaultdict
import os
import django

def driver_open(key_word):
    url = "http://xueshu.baidu.com/"
#     driver = webdriver.PhantomJS("D:/phantomjs-2.1.1-windows/bin/phantomjs.exe")
    driver = webdriver.Chrome("C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe")
    driver.get(url)
    time.sleep(10)
    driver.find_element_by_class_name('s_ipt').send_keys(key_word)
    time.sleep(2)
    driver.find_element_by_class_name('s_btn_wr').click()
    time.sleep(2)
    content = driver.page_source.encode('utf-8')
    driver.close()
    soup = BeautifulSoup(content, 'html.parser')
    return soup

def page_url_list(soup, page=0):
    fir_page = "http://xueshu.baidu.com" + soup.find_all("a", class_="n")[0]["href"]
    urls_list = []
    for i in range(0,75):
        next_page = fir_page.replace("pn=10", "pn={:d}".format(i * 10))
        response = requests.get(next_page)
        soup_new = BeautifulSoup(response.text, "html.parser")
        c_fonts = soup_new.find_all("h3", class_="t c_font")
        for c_font in c_fonts:
            url = c_font.find("a").attrs["href"]
            urls_list.append(url)
            #print(url)
    #print(urls_list)
    return urls_list

def get_item_info(url):
    #print(url)
    content_details = requests.get(url)
    soup = BeautifulSoup(content_details.text, "html.parser")
    # 提取文章题目
    if len(soup.select('#dtl_l > div > h3 > a'))!=0:
        title = ''.join(list(soup.select('#dtl_l > div > h3 > a')[0].stripped_strings))
    elif len(soup.select('#dtl_l > div > h3 > span'))!=0:
        title = ''.join(list(soup.select('#dtl_l > div > h3 > span')[0].stripped_strings))
        title="【图书】"+title
        #print(title)
    else:
        title='none'
    # 提取文章作者
    if len(soup.select('div.author_wr'))!=0:
        authors = ''.join(str(author_) for author_ in list(soup.select('div.author_wr')[0].stripped_strings)[1:])
    else:
        authors="none"
    # 提取摘要
    if len(soup.select('div.abstract_wr p.abstract'))!=0:
        abstract = list(soup.select('div.abstract_wr p.abstract')[0].stripped_strings)[0].replace("\u3000", ' ')
    else:
        abstract="none"
    # 提取年份时间
    if len(soup.select('div.year_wr p.kw_main'))!=0:
        year = list(soup.select('div.year_wr p.kw_main')[0].stripped_strings)[0].replace("\u3000", ' ')
    elif len(soup.select('div.year_wr p.kw_main_s'))!=0:
        year = list(soup.select('div.year_wr p.kw_main_s')[0].stripped_strings)[0].replace("\u3000", ' ')
    elif len(soup.select('div.common_wr p.kw_main_s'))!=0:
        year = list(soup.select('div.common_wr p.kw_main_s')[0].stripped_strings)[0].replace("\u3000", ' ')
        #print(year)
    else:
        year="2007"

    # 提取出版社
    if len(soup.select('div.container_right a.journal_title'))!=0:
        # for i in range(len(soup.select('div.container_right a.journal_title'))):
        #     print(soup.select('div.container_right a.journal_title')[i])
        publish = list(soup.select('div.container_right a.journal_title')[0].stripped_strings)[0].replace("\u3000", ' ')
    elif len(soup.select('div.journal_title'))!=0:
        publish = list(soup.select('div.journal_title')[0].stripped_strings)[0].replace("\u3000", ' ')
    else:
        publish="高等教育出版社"
    ref_wr = list(soup.select('a.sc_cite_cont'))
    if len(ref_wr) == 0:
        ref_wr = 0
    else:
        ref_wr = list(soup.select('a.sc_cite_cont')[0].stripped_strings)[0]
    #提取关键词
    if len(soup.select('div.dtl_search_word > div')) != 0:
        key_words = ','.join(key_word for key_word in list(soup.select('div.dtl_search_word > div')[0].stripped_strings)[1:-1:2])
    else:
        key_words = "None"
#     data = {
#         "title":title,
#         "authors":authors,
#         "abstract":abstract,
#         "year":int(year),
#         "publish":publish,
#         "publish_text":publish_text,
#         "ref_wr":int(ref_wr),
#         "key_words":key_words
#     }
    #print(title)
    return title, authors, abstract, year, publish, ref_wr, key_words


def get_all_data(urls_list):
    dit = defaultdict(list)
    for url in urls_list:
        title, authors, abstract, year, publish, ref_wr, key_words= get_item_info(url)
        dit["title"].append(title)
        dit["authors"].append(authors[0:-3])
        dit["abstract"].append(abstract)
        dit["year"].append(year)
        dit["publish"].append(publish)
        dit["ref_wr"].append(ref_wr)
        dit["key_words"].append(key_words)
        print(title)
        print(url)
        new_paper = Papers()
        new_paper.keywords = key_words
        new_paper.title = title
        #print(new_paper.title)
        new_paper.authors = authors[0:-3]
        new_paper.abstract = abstract
        new_paper.link = url
        new_paper.ref_wf = ref_wr
        new_paper.publish_year = year
        new_paper.publish = publish
        flag = Papers.objects.filter(title=new_paper.title, authors=new_paper.authors).exists()
        if (not flag):
            new_paper.save()
    return dit


# -*- coding:utf-8 -*-
if __name__ == '__main__':
    # -*- coding:utf-8 -*-
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "学术.settings")
    django.setup()
    from search.models import *
    s = "肠,尝,偿,常,厂,场,敞,畅,倡,唱,抄,钞,超,朝,潮, 吵,炒,车,扯,彻,撤,尘,臣,沉,辰,陈,晨,闯,衬,称,趁,撑,成,呈,承, 池,匙,尺,齿,耻,斥,赤,翅,充,冲,诚,城,乘,惩,程,秤,吃,驰,迟,持, 臭,出,初,除,厨,锄,础,储,楚,处,虫,崇,抽,仇,绸,愁,稠,筹,酬,丑, 触,畜,川,穿,传,船,喘,串,疮,窗,床,创,吹,炊,垂,锤,春,纯,唇,蠢, 聪,丛,凑,粗,促,醋,窜,催,摧,脆,词,慈,辞,磁,此,次,刺,从,匆,葱, 大,呆,代,带,待,怠,贷,袋,逮,戴,翠,村,存,寸,错,曾,搭,达,答,打, 蛋,当,挡,党,荡,档,刀,叨,导,岛,丹,单,担,耽,胆,旦,但,诞,弹,淡, 倒,蹈,到,悼,盗,道,稻,得,德,的,灯,登,等,凳,低,堤,滴,敌,笛,底, 抵,地,弟,帝,递,第,颠,典,点,电,店,垫,殿,叼,雕,吊,钓,调,掉,爹, 跌,叠,蝶,丁,叮,盯,钉,顶,订,定,丢,东,冬,董,懂,动,冻,栋,洞,都, 斗,抖,陡,豆,逗,督,毒,读,独,堵,赌,杜,肚,度,渡,端,短,段,断,缎, 朵,躲,惰,鹅,蛾,额,恶,饿,恩,儿,锻,堆,队,对,吨,蹲,盾,顿,多,夺, 而,耳,二,发,乏,伐," \
        "罚,阀,法,帆,番,翻,凡,烦,繁,反,返,犯,泛,饭, 范,贩,方,坊,芳,防,妨,房,仿,访,纺,放,飞,非,肥,匪,废,沸,肺,费, 分,吩,纷,芬,坟,粉,份,奋,愤,粪,丰,风,封,疯,峰,锋,蜂,逢,缝,讽, 凤,奉,佛,否,夫,肤,伏,扶,服,俘,浮,符,幅,福,抚,府,斧,俯,辅,腐, 父,付,妇,负,附,咐,复,赴,副,傅,富,腹,覆,该,改,盖,溉,概,干,甘, 纲,缸,钢,港,杠,高,膏,糕,搞,稿,杆,肝,竿,秆,赶,敢,感,冈,刚,岗, 葛,隔,个,各,给,根,跟,更,耕,工,告,哥,胳,鸽,割,搁,歌,阁,革,格, 弓,公,功,攻,供,宫,恭,躬,巩,共,贡,勾,沟,钩,狗,构,购,够,估,姑, 孤,辜,古,谷,股,骨,鼓,固,故,顾,瓜,刮,挂,乖,拐,怪,关,观,官,冠, 馆,管,贯,惯,灌,罐,光,广,归,龟,规,轨,鬼,柜,贵,桂,跪,滚,棍,锅, 国,果,裹,过,哈,孩,海,害,含,寒,喊,汉,汗,旱,航,毫,豪,好,号,浩, 贺,黑,痕,很,狠,恨,恒,横,衡,轰,耗,喝,禾,合,何,和,河,核,荷,盒, 哄,烘,红,宏,洪,虹,喉,猴,吼,后,厚,候,乎,呼,忽,狐,胡,壶,湖,糊, 化,划,画,话,怀,槐,坏,欢,还,环,蝴,虎,互,户,护,花,华,哗,滑,猾, 晃,谎,灰,恢,挥,辉,回,悔,汇,会,缓,幻,唤,换,患,荒,慌,皇,黄,煌, 活,火,伙,或,货,获,祸,惑,击,饥,绘,贿,惠,毁,慧,昏,婚,浑,魂,混, 吉,级,即,极,急,疾,集,籍,几,己,圾,机,肌,鸡,迹,积,基,绩,激,及, 既,济,继,寄,加,夹,佳,家,嘉,甲,挤,脊,计,记,纪,忌,技,际,剂,季, 间,肩,艰,兼,监,煎,拣,俭,茧,捡,价,驾,架,假,嫁,稼,奸,尖,坚,歼, 健,舰,渐,践,鉴,键,箭,江,姜,将,减,剪,检,简,见,件,建,剑,荐,贱, 郊,娇,浇,骄,胶,椒,焦,蕉,角,狡,浆,僵,疆,讲,奖,桨,匠,降,酱,交, 皆,接,揭,街,节,劫,杰,洁,结,捷,绞,饺,脚,搅,缴,叫,轿,较,教,阶, 今,斤,金,津,筋,仅,紧,谨,锦,尽,截,竭,姐,解,介,戒,届,界,借,巾, 晶,睛,精,井,颈,景,警,净,径,竞,劲,近,进,晋,浸,禁,京,经,茎,惊, 酒,旧,救,就,舅,居,拘,鞠,局,菊,竟,敬,境,静,镜,纠,究,揪,九,久, 橘,举,矩,句,巨,拒,具,俱,剧,惧,据,距,锯,聚,捐,卷,倦,绢,决,绝, 觉,掘,嚼,军,君,均,菌,俊,卡,开,凯,慨,刊,堪,砍,看,康,糠,扛,抗, 炕,考,烤,靠,科,棵,颗,壳,咳,可,渴,克,刻,客,课,肯,垦,恳,坑,空, 孔,恐,控,口,扣,寇,枯,哭,苦,库,裤,酷,夸,垮,挎,跨,块,快,宽,款, 筐,狂,况,旷,矿,框,亏,葵,愧,昆,捆,困,扩,括,阔,垃,拉,啦,喇,腊, 懒,烂,滥,郎,狼,廊,朗,浪,捞,劳,蜡,辣,来,赖,兰,拦,栏,蓝,篮,览, 累,冷,厘,梨,狸,离,犁,鹂,璃,黎,牢,老,姥,涝,乐,勒,雷,垒,泪,类, 励,例,隶,栗,粒,俩,连,帘,怜,莲,礼,李,里,理,力,历,厉,立,丽,利, 梁,粮,粱,两,亮,谅,辆,量,辽,疗,联,廉,镰,脸,练,炼,恋,链,良,凉, 临,淋,伶,灵,岭,铃,陵,零,龄,领,僚,了,料,列,劣,烈,猎,裂,邻,林, 笼,聋,隆,垄,拢,楼,搂,漏,露,芦,令,另,溜,刘,流,留,榴,柳,六,龙, 旅,屡,律,虑,率,绿,卵,乱,掠,略,炉,虏,鲁,陆,录,鹿,滤,碌,路,驴, 落,妈,麻,马,码,蚂,骂,吗,埋,买,轮,论,罗,萝,锣,箩,骡,螺,络,骆, 忙,芒,盲,茫,猫,毛,矛,茅,茂,冒,迈,麦,卖,脉,蛮,馒,瞒,满,慢,漫, 美,妹,门,闷,们,萌,盟,猛,蒙,孟,贸,帽,貌,么,没,眉,梅,煤,霉,每, 棉,免,勉,面,苗,描,秒,妙,庙,灭,梦,迷,谜,米,眯,秘,密,蜜,眠,绵, 摩,磨,魔,抹,末,沫,莫,漠,墨,默,蔑,民,敏,名,明,鸣,命,摸,模,膜, 暮,拿,哪,内,那,纳,乃,奶,耐,男,谋,某,母,亩,木,目,牧,墓,幕,慕, 尼,泥,你,逆,年,念,娘,酿,鸟,尿,南,难,囊,挠,恼,脑,闹,呢,嫩,能, 奴,努,怒,女,暖,挪,欧,偶,辟,趴,捏,您,宁,凝,牛,扭,纽,农,浓,弄, 乓,旁,胖,抛,炮,袍,跑,泡,陪,培,爬,怕,拍,牌,派,攀,盘,判,叛,盼, 碰,批,披,劈,皮,疲,脾,匹,僻,片,赔,佩,配,喷,盆,朋,棚,蓬,膨,捧, 乒,平,评,凭,苹,瓶,萍,坡,泼,婆,偏,篇,骗,漂,飘,票,撇,拼,贫,品, 谱,七,妻,戚,期,欺,漆,齐,其,奇,迫,破,魄,剖,仆,扑,铺,葡,朴,普, 汽,砌,器,恰,洽,千,迁,牵,铅,谦,骑,棋,旗,乞,企,岂,启,起,气,弃, 腔,强,墙,抢,悄,敲,锹,乔,侨,桥,签,前,钱,钳,潜,浅,遣,欠,歉,枪, 禽,勤,青,轻,倾,清,蜻,情,晴,顷,瞧,巧,切,茄,且,窃,亲,侵,芹,琴, 屈,趋,渠,取,去,趣,圈,全,权,泉,请,庆,穷,丘,秋,求,球,区,曲,驱, 群,然,燃,染,嚷,壤,让,饶,扰,绕,拳,犬,劝,券,缺,却,雀,确,鹊,裙, 日,绒,荣,容,熔,融,柔,揉,肉,如,惹,热,人,仁,忍,刃,认,任,扔,仍, 乳,辱,入,软,锐,瑞,润,若,弱,撒,洒,塞,赛,三,伞,散,桑,嗓,丧,扫, 删,衫,闪,陕,扇,善,伤,商,裳,晌,嫂,色,森,杀,沙,纱,傻,筛,晒,山, 哨,舌,蛇,舍,设,社,射,涉,摄,申,赏,上,尚,捎,梢,烧,稍,勺,少,绍, 慎,升,生,声,牲,胜,绳," \
        "省,圣,盛,伸,身,深,神,沈,审,婶,肾,甚,渗, 石,时,识,实,拾,蚀,食,史,使,始,剩,尸,失,师,诗,施,狮,湿,十,什, 视,试,饰,室,是,柿,适,逝,释,誓,驶,士,氏,世,市,示,式,事,侍,势, 书,叔,殊,梳,疏,舒,输,蔬,熟,暑,收,手,守,首,寿,受,兽,售,授,瘦, 耍,衰,摔,甩,帅,拴,双,霜,爽,谁,鼠,属,薯,术,束,述,树,竖,数,刷, 斯,撕,死,四,寺,似,饲,肆,松,宋,水,税,睡,顺,说,嗽,丝,司,私,思, 速,宿,塑,酸,蒜,算,虽,随,岁,碎,诵,送,颂,搜,艘,苏,俗,诉,肃,素, 它,塌,塔,踏,台,抬,太,态,泰,贪,穗,孙,损,笋,缩,所,索,锁,她,他, 摊,滩,坛,谈,痰,毯,叹,炭,探,锻,堆,队,对,吨,蹲,盾,顿,多,夺, 涛,掏,滔,逃,桃,陶,淘,萄,讨,套,汤,唐,堂,塘,膛,糖,倘,躺,烫,趟, 惕,替,天,添,田,甜,填,挑,条,跳,特,疼,腾,梯,踢,提,题,蹄,体,剃, 通,同,桐,铜,童,统,桶,筒,痛,偷,贴,铁,帖,厅,听,亭,庭,停,挺,艇, 土,吐,兔,团,推,腿,退,吞,屯,托,头,投,透,秃,突,图,徒,涂,途,屠, 外,弯,湾,丸,完,玩,顽,挽,晚,碗,拖,脱,驼,妥,娃,挖,蛙,瓦,袜,歪, 危,威,微,为,围,违,唯,维,伟,伪,万,汪,亡,王,网,往,妄,忘,旺,望, 温,文,纹,闻,蚊,稳,问,翁,窝,我,尾,委,卫,未,位,味,畏,胃,喂,慰, 午,伍,武,侮,舞,勿,务,物,误,悟,沃,卧,握,乌,污,呜,屋,无,吴,五, 稀,溪,锡,熄,膝,习,席,袭,洗,喜,雾,夕,西,吸,希,析,息,牺,悉,惜, 吓,夏,厦,仙,先,纤,掀,鲜,闲,弦,戏,系,细,隙,虾,瞎,峡,狭,霞,下, 宪,陷,馅,羡,献,乡,相,香,箱,详,贤,咸,衔,嫌,显,险,县,现,线,限, 削,宵,消,销,小,晓,孝,效,校,笑,祥,享,响,想,向,巷,项,象,像,橡, 泻,卸,屑,械,谢,心,辛,欣,新,薪,些,歇,协,邪,胁,斜,携,鞋,写,泄, 姓,幸,性,凶,兄,胸,雄,熊,休,修,信,兴,星,腥,刑,行,形,型,醒,杏, 许,序,叙,绪,续,絮,蓄,宣,悬,旋,羞,朽,秀,绣,袖,锈,须,虚,需,徐, 训,讯,迅,压,呀,押,鸦,鸭,牙,芽,选,穴,学,雪,血,寻,巡,旬,询,循, 岩,沿,炎,研,盐,蜒,颜,掩,眼,演,崖,哑,雅,亚,咽,烟,淹,延,严,言, 扬,羊,阳,杨,洋,仰,养,氧,痒,样,厌,宴,艳,验,焰,雁,燕,央,殃,秧, 耀,爷,也,冶,野,业,叶,页,夜,液,妖,腰,邀,窑,谣,摇,遥,咬,药,要, 乙,已,以,蚁,倚,椅,义,亿,忆,艺,一,衣,医,依,仪,宜,姨,移,遗,疑, 毅,翼,因,阴,姻,音,银,引,饮,隐,议,亦,异,役,译,易,疫,益,谊,意, 影,映,硬,佣,拥,庸,永,咏,泳,勇,印,应,英,樱,鹰,迎,盈,营,蝇,赢, 游,友,有,又,右,幼,诱,于,予,余,涌,用,优,忧,悠,尤,由,犹,邮,油, 雨,语,玉,育,郁,狱,浴,预,域,欲,鱼,娱,渔,愉,愚,榆,与,宇,屿,羽, 圆,援,缘,源,远,怨,院,愿,约,月,御,裕,遇,愈,誉,冤,元,员,园,原, 晕,韵,杂,灾,栽,宰,载,再,在,咱,钥,悦,阅,跃,越,云,匀,允,孕,运, 皂,造,燥,躁,则,择,泽,责,贼,怎,暂,赞,脏,葬,遭,糟,早,枣,澡,灶, 宅,窄,债,寨,沾,粘,斩,展,盏,崭,增,赠,渣,扎,轧,闸,眨,炸,榨,摘, 胀,障,招,找,召,兆,赵,照,罩,遮,占,战,站,张,章,涨,掌,丈,仗,帐, 诊,枕,阵,振,镇,震,争,征,挣,睁,折,哲,者,这,浙,贞,针,侦,珍,真, 汁,芝,枝,知,织,肢,脂,蜘,执,侄,筝,蒸,整,正,证,郑,政,症,之,支, 指,至,志,制,帜,治,质,秩,致,智,直,值,职,植,殖,止,只,旨,址,纸, 舟,周,洲,粥,宙,昼,皱,骤,朱,株,置,中,忠,终,钟,肿,种,众,重,州, 住,助,注,驻,柱,祝,著,筑,铸,抓,珠,诸,猪,蛛,竹,烛,逐,主,煮,嘱, 追,准,捉,桌,浊,啄,着,仔,姿,资,爪,专,砖,转,赚,庄,装,壮,状,撞, 走,奏,租,足,族,阻,组,祖,钻,嘴,滋,子,紫,字,自,宗,棕,踪,总,纵, 最,罪,醉,尊,遵,昨,左,作,坐,座,做, 办,半,伴,扮,拌,瓣,帮,绑,榜,膀,傍,棒,包,胞,雹,宝,饱,保,堡,报, 抱,暴,爆,杯,悲,碑,北,贝,备,背,倍,被,辈,奔,本,笨,蹦,逼,鼻,比, 彼,笔,鄙,币,必,毕,闭,毙,弊,碧,蔽,壁,避,臂,边,编,鞭,扁,便,变, 辫,辨,辩,辪,辬,表,别,宾,滨,冰,兵,丙,柄,饼,并,病,拨,波,玻,剥, 脖,菠,播,伯,驳,泊,博,搏,膊,薄,卜,补,捕,不,布,步,怖,部,擦,猜, 才,材,财,裁,采,彩,睬,踩,菜,参,餐,残,蚕,惭,惨,灿,仓,苍,舱,藏, 操,槽,草,册,侧,厕,测,策,层,叉,插,查,茶,察,岔,差,拆,柴,馋,缠, 产,铲,颤,昌,长,肠,尝,偿,常,厂,场,敞,畅,倡,唱,抄,钞,超,朝,潮, 吵,炒,车,扯,彻,撤,尘,臣,沉,辰,陈,晨,闯,衬,称,趁,撑,成,呈,承, 池,匙,尺,齿,耻,斥,赤,翅,充,冲,诚,城,乘,惩,程,秤,吃,驰,迟,持, 臭,出,初,除,厨,锄,础,储," \
        "楚,处,虫,崇,抽,仇,绸,愁,稠,筹,酬,丑, 触,畜,川,穿,传,船,喘,串,疮,窗,床,创,吹,炊,垂,锤,春,纯,唇,蠢, 聪,丛,凑,粗,促,醋,窜,催,摧,脆,词,慈,辞,磁,此,次,刺,从,匆,葱, 大,呆,代,带,待,怠,贷,袋,逮,戴,翠,村,存,寸,错,曾,搭,达,答,打, 蛋,当,挡,党,荡,档,刀,叨,导,岛,丹,单,担,耽,胆,旦,但,诞,弹,淡, 倒,蹈,到,悼,盗,道,稻,得,德,的,灯,登,等,凳,低,堤,滴,敌,笛,底, 抵,地,弟,帝,递,第,颠,典,点,电,店,垫,殿,"
    slist = (str(s).split(","))

    for i in range(0,5):
        soup = driver_open(slist[i])
        urls_list = page_url_list(soup, page=0)
        dit = get_all_data(urls_list)
    # soup = driver_open("马克思恩格斯选集")
    # urls_list = page_url_list(soup, page=0)
    # dit = get_all_data(urls_list)






